# Purpose: Produce the robustness exercises in Appendix L
source("regional analysis/code/general_friendliness_validation_helpers.R") # A set of helper functions

#####################################
## 1. Read in and prepare the data ##
#####################################

# See README for instructions on how to access this data
german_immigration_sentiment_nuts3 = read_csv(german_immigration_sentiment_nuts_fp(3)) %>%
  rename(NUTS_ID = nuts3_region_2016)

german_immigration_sentiment_nuts2 = read_csv(german_immigration_sentiment_nuts_fp(2)) %>%
  rename(NUTS_ID = nuts2_region_2016)

fb_int_nuts3 <- read_dta(fb_integration_fp)

fb_int_nuts1 <- fb_int_nuts3 %>%
  select(
    nuts3,
    pop_size = pop_for_tot_2010,
    n_frnd_nat_lcl_sy_avg_re,
    n_frnd_nat_lcl_sy_n,
    n_frnd_nat_lcl_native_avg_re,
    n_frnd_nat_lcl_native_n,
    produ_any_de_sy_avg_re,
    produ_any_de_sy_n,
    n_frnd_nat_lcl_sy_nat_avg_re
  ) %>%
  mutate(nuts1 = str_sub(nuts3, end = -3)) %>% 
  group_by(nuts1) %>%
  summarise(
    pop_size = sum(pop_size),
    friending_integration = weighted.mean(n_frnd_nat_lcl_sy_avg_re, n_frnd_nat_lcl_sy_n),
    general_friending = weighted.mean(n_frnd_nat_lcl_native_avg_re, n_frnd_nat_lcl_native_n),
    syrians_producing_german_content = weighted.mean(produ_any_de_sy_avg_re, produ_any_de_sy_n),
    relative_friending = weighted.mean(n_frnd_nat_lcl_sy_nat_avg_re, n_frnd_nat_lcl_sy_n) 
  )

fb_int_nuts2 <- fb_int_nuts3 %>%
  select(
    nuts3,
    pop_size = pop_for_tot_2010,
    n_frnd_nat_lcl_sy_avg_re,
    n_frnd_nat_lcl_sy_n,
    n_frnd_nat_lcl_native_avg_re,
    n_frnd_nat_lcl_native_n,
    produ_any_de_sy_avg_re,
    produ_any_de_sy_n,
    n_frnd_nat_lcl_sy_nat_avg_re
  ) %>%
  mutate(nuts2 = str_sub(nuts3, end = -2)) %>% 
  group_by(nuts2) %>%
  summarise(
    pop_size = sum(pop_size),
    friending_integration = weighted.mean(n_frnd_nat_lcl_sy_avg_re, n_frnd_nat_lcl_sy_n),
    general_friending = weighted.mean(n_frnd_nat_lcl_native_avg_re, n_frnd_nat_lcl_native_n),
    syrians_producing_german_content = weighted.mean(produ_any_de_sy_avg_re, produ_any_de_sy_n),
    relative_friending = weighted.mean(n_frnd_nat_lcl_sy_nat_avg_re, n_frnd_nat_lcl_sy_n) 
  )

correlation_info_nuts2 = german_immigration_sentiment_nuts2 %>%
  inner_join(fb_int_nuts2, by = join_by(NUTS_ID == nuts2)) %>%
  rename(wvs_sample_size = n_respondents) %>%
  mutate(
    trust_people_meet_first_time_percent_positive = percent_trust_strangers,
    trust_people_meet_first_time_percent_negative = 1 - trust_people_meet_first_time_percent_positive 
  ) %>% 
  select(
    NUTS_ID,
    pop_size,
    wvs_sample_size,
    general_friending,
    trust_people_meet_first_time_percent_positive,
    trust_people_meet_first_time_percent_negative,
  )


correlation_info_nuts3 = german_immigration_sentiment_nuts3 %>%
  inner_join(fb_int_nuts3, by = join_by(NUTS_ID == nuts3)) %>%
  rename(
    pop_size = pop_for_tot_2010,
    wvs_sample_size = n_respondents,
    friending_integration = n_frnd_nat_lcl_sy_avg_re,
    general_friending = n_frnd_nat_lcl_native_avg_re,
    syrians_producing_german_content = produ_any_de_sy_avg_re,
    relative_friending = n_frnd_nat_lcl_sy_nat_avg_re
  ) %>%
  mutate(
    trust_people_meet_first_time_percent_positive = percent_trust_strangers,
    trust_people_meet_first_time_percent_negative = 1 - trust_people_meet_first_time_percent_positive 
  ) %>% 
  select(
    NUTS_ID,
    pop_size,
    wvs_sample_size,
    general_friending,
    trust_people_meet_first_time_percent_positive,
    trust_people_meet_first_time_percent_negative,
  )


#################################
### 2. Make the scatter plots ###
#################################

correlation_info_nuts2 %>% rename(region = NUTS_ID) %>% create_and_save_scatter(
  "trust_people_meet_first_time_percent_positive",
  "% Trust Person When Meeting Someone New",
  "evs5_nuts2"
)

correlation_info_nuts3 %>% rename(region = NUTS_ID) %>% create_and_save_scatter(
  "trust_people_meet_first_time_percent_positive",
  "% Trust Person When Meeting Someone New",
  "evs5_nuts3"
)

ess_8 <- read_csv(ess_8_fp)
ess_9 <- read_csv(ess_9_fp)

ess_8_9 <- bind_rows(ess_8, ess_9)
ess_8_9 %>%
  create_data_for_correlation() %>%
  create_friendliness_plots("ess_8_9_pooled")
